package com.yong;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;

import java.io.IOException;
import java.math.BigDecimal;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

/**
 * 描述:爬取京东数据
 *
 * @author BluceYoung
 * @create 2020-06-30 17:21
 * B站：TeacherYoung凌风
 * www.teacheryoung.com
 */
@SpringBootTest
public class JDCrawler {

    @Autowired
    private RestHighLevelClient restHighLevelClient;

    @Test
    void doCrawler() throws IOException {
        String keyword = "柳传志";
        keyword = URLEncoder.encode(keyword, "UTF-8");//处理中文
        String url = "https://search.jd.com/Search?keyword=" + keyword;

        Document document = Jsoup.parse(new URL(url), 30000);

        //获取商品列表的div
        Element j_goodsList = document.getElementById("J_goodsList");

        //获取所有的li集合，每个li是一个商品
        Elements li = j_goodsList.getElementsByTag("li");

        List<JDGoods> list = new ArrayList<JDGoods>();
        for (Element element : li) {
            list.add(new JDGoods(element));
        }

        insert2ES(list);
    }

    /**
     * 把数据存入es的jd_goods索引
     * @param list
     */
    void insert2ES(List<JDGoods> list) throws IOException {

        BulkRequest bulkRequest = new BulkRequest();
        ObjectMapper mapper = new ObjectMapper();

        for (JDGoods jdGoods : list) {
            bulkRequest.add(
                    new IndexRequest("jd_goods")
                    .source(mapper.writeValueAsString(jdGoods), XContentType.JSON)
            );
        }
        BulkResponse response = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
       if(!response.hasFailures()){
           System.out.println("成功插入"+list.size()+"条数据");
       }else{
           System.out.println("失败");
       }
    }
}

class JDGoods {
    private String title;
    private String src;
    private BigDecimal price;

    /**
     * 根据li标签构造一本书
     *
     * @param li
     */
    public JDGoods(Element li) {
        Element eleName = li.getElementsByClass("p-name").get(0).getElementsByTag("em").get(0);
        this.title = eleName.text();

        Element eleImg = li.getElementsByClass("p-img").get(0).getElementsByTag("img").get(0);
        this.src = eleImg.attr("src");

        Element elePrice = li.getElementsByClass("p-price").get(0).getElementsByTag("i").get(0);
        this.price = new BigDecimal(elePrice.text());
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }

    public String getSrc() {
        return src;
    }

    public void setSrc(String src) {
        this.src = src;
    }

    public BigDecimal getPrice() {
        return price;
    }

    public void setPrice(BigDecimal price) {
        this.price = price;
    }
}


